package com.cloud.spider;

import org.jsoup.Connection;
import org.jsoup.Jsoup;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;

public class GitHubSpider {

    static String targetFormat = "https://api.github.com/search/repositories?q=language:%s&sort=stars&page=%d&per_page=%d";
    static int perPage = 100;

    public static void main(String[] args) throws InterruptedException {
        // go,c,r,ruby,rust,swift
        String lang = "php";
        spiderEachLang(lang);
    }

    public static void spiderEachLang(String lang) {
        int startPage = 1;
        int offset = 10;
        File saveDir = new File("data/" + lang);
        if (!saveDir.exists()) saveDir.mkdirs();
        for (int i = 0; i < offset; i++) {
            int pageNo = startPage + i;
            String target = String.format(targetFormat, lang, pageNo, perPage);
            Connection conn = Jsoup.connect(target).ignoreContentType(true);
            try {
                Connection.Response resp = conn.execute();
                String body = resp.body();
                File saveData = new File("data/" + lang + "/data_" + pageNo + ".json");
                BufferedWriter bw = new BufferedWriter(new FileWriter(saveData));
                bw.write(body);
                bw.flush();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

}
